package com.asiainfo.zqx;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import org.jsoup.select.Elements;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;

public class Xinlangcaijin extends BreadthCrawler {
    private String seedurl="https://finance.sina.com.cn/";
    private String regurl="https://finance.sina.com.cn/.*/2022.*.shtml";
    public Xinlangcaijin(String crawlPath) {
        super(crawlPath, false);
        addSeed(seedurl);
        setThreads(1);
        //setResumable(true);
    }

    @Override
    public void visit(Page page, CrawlDatums crawlDatums) {
        if(page.matchType("text")&&page.select("span.source.ent-source").text().contains("新浪财经")){
            String url=page.url();
            String title=page.select("div.main-content.w1240>h1").text();
            String source="新浪财经";
            String time=page.select("span.date").text();
            //转为时间戳
            SimpleDateFormat formatter = new SimpleDateFormat("yyyy年MM月dd日 HH:mm", Locale.getDefault());
            //转为2022-1-1 11:11:11时间格式
            SimpleDateFormat righttime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.getDefault());
            Date dateString = null;
            try {
                dateString = formatter.parse(time);
                time = righttime.format(dateString);
            } catch (ParseException e) {
                e.printStackTrace();
            }
            String writer=page.select("span.source.ent-source").text();
            String  text=page.select("div.article>p").text();
            System.out.println(url);
            System.out.println(title);
            System.out.println(time);
            System.out.println(writer);
            System.out.println(text);

        }else {
            Elements elements=page.select("a");
            for(int i=0;i<elements.size();i++){
                if(elements.get(i).attr("abs:href").matches(regurl)){
                    crawlDatums.add(new CrawlDatum(elements.get(i).attr("abs:href"),"text"));
                }
            }

        }
    }

    public static void main(String[] args) throws Exception {
       Xinlangcaijin xinl=new Xinlangcaijin("xinla");
       xinl.start(2);
    }
}
